package de.uni_saarland.coli.doclists.texttools;

import de.uni_saarland.coli.doclists.document.ITextDocument;

/**
 * 
 * @author stefan
 * 
 *         text tokenizer
 */
public interface ITextTokenizer {
	/**
	 * Has next.
	 * 
	 * @return A <code>boolean</code>: <br>
	 *         true : if more tokens exist <br>
	 *         false : otherwise
	 */
	boolean hasNext();

	/**
	 * Next token.
	 * 
	 * @return A <code>int</code> representing the next token.
	 */
	public int next();

	public void startDocument(ITextDocument doc);

	public void startText(String text);

}
